package com.itheima.webmagic.crawlerTask;

import com.itheima.webmagic.pojo.JobInfo;
import com.itheima.webmagic.utils.MathSalary;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

@Component
public class JobProcessor implements PageProcessor {

    @Autowired
    private SpringDataPipeline springDataPipeline;

    //@Scheduled(initialDelay = 1000, fixedDelay = 1000 * 10000)
    public void process() {
        //访问入口url地址
        //String url = "https://search.51job.com/list/000000,000000,0000,01%252C32,9,99,java,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=";
        //String url = "https://search.51job.com/list/000000,000000,0000,32%252C01,9,99,java,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=";
        String url = "https://search.51job.com/list/000000,000000,0000,32%252C01,9,99,%25E6%25B5%258B%25E8%25AF%2595%25E5%25BC%2580%25E5%258F%2591,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=";
        Spider.create(new JobProcessor())
                .addUrl(url) 
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(10000000)))
                .addPipeline(this.springDataPipeline)
                .thread(5)
                .run();
    }

    @Override
    public void process(Page page) {
        //获取页面数据
        List<Selectable> nodes = page.getHtml().$("div#resultList div.el").nodes();

        //判断nodes是否为空
        if (nodes.isEmpty()) {
            try {
                //如果为空，表示这是招聘信息详情页保存信息详情
                this.saveJobInfo(page);
            } catch (Exception e) {
                e.printStackTrace();
            }

        } else {
            //如果有值，表示这是招聘信息列表页
            for (Selectable node : nodes) {
                //获取招聘信息详情页url
                String jobUrl = node.links().toString();
                //添加到url任务列表中，等待下载
                page.addTargetRequest(jobUrl);

                //获取翻页按钮的超链接
                List<String> listUrl = page.getHtml().$("div.p_in li.bk").links().all();
                //添加到任务列表中
                page.addTargetRequests(listUrl);

            }
        }
    }

    private Site site = Site.me()/*.setCharset("UTF-8")*///编码
            .setSleepTime(1)//抓取间隔时间
            .setTimeOut(1000*10)//超时时间
            .setRetrySleepTime(3000)//重试时间
            .setRetryTimes(3);//重试次数
    public Site getSite() {
        return site;
    }


    /**
     * 解析页面，获取招聘详情
     *
     * @param
     */
    private void saveJobInfo(Page page) {
        //创建招聘信息对象
        JobInfo jobInfo = new JobInfo();
        Html html = page.getHtml();

        //公司名称
        jobInfo.setCompanyName(html.$("div.tHeader p.cname a", "text").toString());
        //公司地址
        //jobInfo.setCompanyAddr(html.$("div.tBorderTop_box:nth-child(3) p.fp", "text").toString());
        jobInfo.setCompanyAddr(html.$("div.tCompany_main > div.tBorderTop_box > div.bmsg > p.fp", "text").toString());
        //公司信息
        jobInfo.setCompanyInfo(html.$("div.tmsg", "text").toString());
        //职位名称
        jobInfo.setJobName(html.$("div.tHeader > div.in > div.cn > h1", "text").toString());
        //工作地点
        //jobInfo.setJobAddr(html.$("div.tCompany_main > div.tBorderTop_box > div.bmsg > p.fp", "text").toString());
        String jobaddr = html.$("div.tHeader > div.in >div.cn > p.msg", "title").toString();
        if(StringUtils.isNoneEmpty(jobaddr)) {
            jobaddr = StringUtils.substringBefore(jobaddr, "|");
            jobInfo.setJobAddr(jobaddr.substring(0, jobaddr.length() - 2));
        }
        //职位信息
        jobInfo.setJobInfo(Jsoup.parse(html.$("div.tCompany_main div.tBorderTop_box:nth-child(1)").toString()).text());
        //工资范围
        String salaryStr = html.$("div.tHeader > div.in > div.cn > strong", "text").toString();
        System.out.println("----------------------------"+salaryStr);
        if(StringUtils.isNoneEmpty(salaryStr)) {
            jobInfo.setSalaryMin(MathSalary.getSalary(salaryStr)[0]);
            jobInfo.setSalaryMax(MathSalary.getSalary(salaryStr)[1]);
        }
        //职位详情url
        jobInfo.setUrl(page.getUrl().toString());
        //职位发布时间
        //String time = html.$("div.jtag > div.t1 > span.sp4", "text").regex(".*发布").toString();
        String time = html.$("div.tHeader > div.in >div.cn > p.msg", "title").toString();
        time=StringUtils.substringBefore(time, "发布");
        time=StringUtils.substringAfterLast(time, "|");
        //time=time.substring(2, time.length());
        jobInfo.setTime(time.substring(2, time.length()));
        //jobInfo.setTime(time.substring(0, time.length() - 2));

        //保存数据
        page.putField("jobInfo", jobInfo);
    }
}